#ifndef TOKENIZER_H
#define TOKENIZER_H
#include<QString>
#include<QChar>
#include<QTextStream>
class Tokenizer
{
	public:
		Tokenizer();
		Tokenizer(const QString filename,QTextStream& fp);
		~Tokenizer();
		void SkipComment();
		bool IsEOF()
		{
			return m_TokenIndex>m_BufferLen;
		}

		bool NotEOF()
		{
			return m_TokenIndex<m_BufferLen;
		}

		QChar CurrentChar()
		{
			return m_Buffer.at(m_TokenIndex);
		}

		QChar NextChar()
		{
			if(IsEOF())
				return QChar('\0');
			return m_Buffer.at(m_TokenIndex+1);
		}
	private:
		int m_TokenIndex;
		QString m_FileName;
		QString m_Buffer;
		QString m_BufferLen;
		int m_LineNum;
		int m_ColoNum;  //列号
		
};

#endif
